package com.bestsonic.crawler.utils;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;

import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;

public class URLUtils {

	public static String getContent(String url){
		CloseableHttpClient client = HttpClients.createDefault();
		HttpGet get = new HttpGet(url);
		//WebClient webClient = new WebClient();
		//WebResponse response = null;
		InputStream in = null;
		BufferedReader buff = null;
		StringBuilder sb = new StringBuilder();
		String str = null;
		try{
			//URLConnection conn = new URL(url).openConnection();
			get.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
			get.addHeader("Accept-Language", "zh-CN,zh;q=0.8");
			get.addHeader("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20131029 Firefox/17.0");
			get.addHeader("DNT", "1");
			get.addHeader("Cache-Control", "max-age=0");
//			webClient.getOptions().setActiveXNative(false);
//			webClient.getOptions().setAppletEnabled(false);
//			webClient.getOptions().setCssEnabled(false);
//			webClient.getOptions().setDoNotTrackEnabled(false);
//			webClient.getOptions().setGeolocationEnabled(false);
//			webClient.getOptions().setJavaScriptEnabled(true);
//			webClient.getOptions().setPopupBlockerEnabled(false);
//			webClient.getOptions().setThrowExceptionOnScriptError(false);
//			webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
//			webClient.getOptions().setRedirectEnabled(true);
//			webClient.getOptions().setPrintContentOnFailingStatusCode(true);
//			response = webClient.getPage(url).getWebResponse();
			//in = conn.getInputStream();
			//in = response.getContentAsStream();
			in = client.execute(get).getEntity().getContent();
			buff = new BufferedReader(new InputStreamReader(in));
			while((str = buff.readLine()) != null){
				sb.append(str+"\r\n");
			}
		} catch (IOException e) {
			e.printStackTrace();
		} 
		
		finally{
			StreamUtils.closeStream(buff);
			StreamUtils.closeStream(in);
		}
		return sb.toString();
	}
}
